import com.db4o.Db4o;
import com.db4o.ObjectContainer;
import com.db4o.ObjectSet;
import edu.uci.ics.crawler4j.crawler.CrawlController;
import edu.uci.ics.crawler4j.crawler.Page;

import java.util.List;

public class Controller {
        public static ObjectContainer db = null;
		public static void main(String[] args) throws Exception {

            try {
                db = Db4o.openFile("occupy.data");
             } catch (Exception e) {
                e.printStackTrace();
             }

			String rootFolder = "C:\\Users\\n644dc\\crawler\\dump";
			int numberOfCrawlers = 16;
			CrawlController controller = new CrawlController(rootFolder);
            controller.addSeed("http://www.google.com/search?aq=2&hl=en&gl=us&tbm=nws&btnmeta_news_search=1&q=occupy");
            controller.addSeed("http://www.reuters.com");
            controller.addSeed("http://www.bloomberg.com/");
            controller.addSeed("http://www.usatoday.com");
            controller.addSeed("http://www.bbc.co.uk");
			controller.addSeed("http://www.cnn.com");
            controller.addSeed("http://www.cnn.com/US");
			controller.addSeed("http://www.bbc.co.uk/news/world/us_and_canada/");
			controller.addSeed("http://www.reuters.com/news/us");



			controller.setPolitenessDelay(100);

			controller.setMaximumCrawlDepth(-1);

			controller.setMaximumPagesToFetch(1000000);

			controller.start(Crawler.class, numberOfCrawlers);
            System.out.println("****-----****----**** Crawl Done **** ----- ******");
            printDB();
            System.out.println("****-----****----**** EXIT **** ----- ******");
            if (db != null)
                db.close();
		}
    public static void printDB() {
        List<Page> pages = null;
        try {
            pages = db.query(Page.class);
        } catch (Exception e) {
            e.printStackTrace();
        }
        if (pages.size()>0) {
            System.out.println("-------Number of pages>>>>> "+pages.size()+" <<<<<<<---------");
            for (Page page : pages) {
                System.out.println(page.getTitle()+"  -   url["+page.getWebURL().getURL()+"]");
            }
        }
    }
}
